
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

import java.io.*;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;

/**
 * Created by John on 2019/07/31 16:24.
 */
public class HdfsBatchDel {

    private static List<String> excludePath = Arrays.asList(new String[]{"/", "/user", "/user/hive/warehouse", "/user/hive/warehouse/"});

    public static void main(String[] args) {
        if (args.length < 1) {
            System.out.println("Usage: HdfsBatchDel <localFilePath>");
            System.exit(1);
        }
        System.out.println("start hdfs oper!");
        long start = System.currentTimeMillis();
        try {
            FileSystem fileSystem = FileSystem.newInstance(new Configuration());
            int cnt = 0;
            List<String> lines = getLines(args[0]);
            if (lines == null || lines.size() == 0) {
                System.out.println("invalid file!!!");
                System.exit(1);
            }
            for (String line : lines) {
                if(excludePath.contains(line)){
                    System.out.println("*** special path: " + line);
                    continue;
                }
                fileSystem.delete(new Path(line), true);
                ++cnt;
                if (cnt % 1000 == 0) {
                    System.out.println("cnt = " + cnt);
                }
            }
            System.out.println("total cnt = " + cnt);
            fileSystem.close();
        } catch (IOException e) {
            e.printStackTrace();
        }

        System.out.println("finish hdfs oper!");
        System.out.println("total used time: " + 1.0 * (System.currentTimeMillis() - start) / 1000 + " second(s)");
    }

    private static List<String> getLines(String path) {
        List<String> list = new ArrayList<String>();
        try {
            InputStream fileInput = new FileInputStream(new File(path));
            InputStreamReader input = new InputStreamReader(fileInput, Charset.defaultCharset());
            BufferedReader reader = toBufferedReader(input);
//            list =
            String line = reader.readLine();
            while (line != null) {
                list.add(line);
                line = reader.readLine();
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
        return list;
    }

    private static BufferedReader toBufferedReader(Reader reader) {
        return reader instanceof BufferedReader ? (BufferedReader) reader : new BufferedReader(reader);
    }

    public static int add(int a, int b) {
        return a + b;
    }
}
